/*
 * This program crawls across IMDB to scrape information to populate the movie
 * database for our EECS 341 project.
 */
package eecs341crawler;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * Add your name if you edit
 * @author David Kent
 */
public class Main {

    ArrayList<String> names, titles;  //names of people and titles of movies that have not yet been searched
    ArrayList<String> researchedNames, researchedTitles;  //names of people and titles of movies that have been searched.

    public Main() {
        //declare all variables
        names = new ArrayList<String>();
        titles = new ArrayList<String>();
        researchedNames = new ArrayList<String>();
        researchedTitles = new ArrayList<String>();
        //clear file from last use
        printToFile("", false);
    }

    public static void main(String[] args) throws IOException {
        //Requisite main method.  Just creats instance of Main and calls run
        Main m = new Main();
        m.run(2, "tt0844286");
    }

    /**
     * Controller method for trawling through IMDB.  It gets information from a
     * movie page, then uses the names of actors it finds to get more movie
     * titles.
     * @param iter the number of movies to search through
     * @param seed IMDB movie id.  Ex: tt0844286
     */
    public void run(int iter, String seed) {
        titles.add(seed);  //use seed to populate titles, since it might be empty
        for (int i = 0; i < iter; i++) {  //Get information for iter number of movies
            if (titles.size() == 0) {
                break;  //Since we start with a seed, this only occurs if we've also run out of names
            }
            String movieID = titles.remove(0);  //Remove first movie.  We could also randomly choose a movie
            researchedTitles.add(movieID);  //Add to researchedTitles so we don't accidentally look it up twice
            String html = getHTML(movieID); //Get the HTML source code from the web
            if (html == null) {
                continue;  //If nothing was returned for some reason, just continue on
            }
            getMovieInfo(html, movieID); //Scrape information about the movie
            while (titles.size() == 0 && names.size() > 0) {  //If we need more movies to research and we have names of actors to look from
                findMovies();  //then find more movies
            }
        }
    }

    /**
     * Gets the HTML source code for a given IMDB id
     * @param id The id for a person's name (e.g. "nm0426059") or movie (e.g. "tt0844286")
     * @return the source code for the IMDB website as a String.  Null if something went wrong.
     */
    public String getHTML(String id) {
        String html = null;  //Default value to return, indicates an error occurred
        String type = "";  //Used to distinguish between movie titles and person's name
        if (id.startsWith("tt")) //We have a title
        {
            type = "title";
        } else if (id.startsWith("nm")) //We have a name
        {
            type = "name";
        } else { //An error occurred
            System.out.println("ERROR: Unsure of id type: " + id);
            return null;
        }

        try {
            WebFile file = new WebFile("http://www.imdb.com/" + type + "/" + id + "/"); //Create URL and make a WebFile
            String MIME = file.getMIMEType();
            Object content = file.getContent();
            if (MIME.equals("text/html") && content instanceof String) {
                html = (String) content;  //Everything went fine and we got the source code from WebFile
            }
        } catch (MalformedURLException ex) {
            Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(Main.class.getName()).log(Level.SEVERE, null, ex);
        }
        return html;
    }


    /**
     * Given an actor's id, get their basic information and write a statement to
     * add them to the database
     * @param id The IMDB id for an actor
     * @return true if the person was added, false otherwise
     */
    public boolean addPerson(String id) {
        String html = getHTML(id);  //get the html source for this actor
        System.out.println("id: " + id);  //Useful debug code we we know stuff is happenning.
        if (html == null) { //An error occurred
            return false;
        }
        String name = "", dob = "", dod = "";  //Actor's name, date of birth, and date of death
        int i = html.indexOf("<title>") + 7; //Person's name is the title to the webpage
        int j = html.indexOf("</title>");
        name = html.substring(i, j);

        dob = dateOfBirth(html, true);  //get date of birth
        dod = dateOfBirth(html, false); //get date of death
        if (dob == null || dod == null || dob.equals("")) { //check for any errors
            return false;
        }
        printToFile("person(" + id + "," + name + ",[" + dob + "],[" + dod + "])\n", true);  //Add statement ot add actor to database
        return true;  //Everything went well if we got here
    }

    /**
     * Get the Date of Birth or Death of an actor
     * @param html the html source for an actor's IMDB page
     * @param birth true if finding date of birth, false if finding date of death
     * @return Date in the form of "Day Month, Year", null if error, "" if none (for death only)
     */
    public String dateOfBirth(String html, boolean birth) {
        String type = birth ? "Birth" : "Death";  //Decide type of date we're trying to find
        String date;
        int i = html.indexOf("Date of " + type);  //This is how it's formatted in the IMDB page
        if (i == -1) //no date of that type
        {
            if (birth) //should always have a birthday
            {
                return null;
            } else //possible they're still alive
            {
                return "none";
            }
        }
        i = html.indexOf("<a href=\"/date", i);  //I think all dates are linked this way
        if (i == -1) {
            return null; //no specific day, error for both birth and death.
                         //Happens if only birth year is given or nothing is provided.
                         //For our convenience, we take only actors with a specific birth and death day
        }
        int j = html.indexOf("/a>", i) + 3;  //Find the date
        String s = html.substring(i, j);
        date = s.replaceAll("<.*?>", "");  //Something of the form "Day Month"
        i = html.indexOf("<a href=\"/search/name?" + type.toLowerCase() + "_year=", i); //Find the year
        if (i == -1) {
            return null;  //no year given, not sure if this ever actually happens
        }
        j = html.indexOf("/a>", i) + 3;
        s = html.substring(i, j);
        date += ", " + s.replaceAll("<.*?>", "");  //Tack on year to date
        return date;
    }

    /**
     * Get all bunch of the relations concerning movies
     * @param html  IMDB html source code for a particular movie
     * @param movieID the IMDB id for this movie, e.g. "tt0844286"
     */
    public void getMovieInfo(String html, String movieID) {
        String title = "", year = "", language = "", type = "movie";  //This only scrapes a movie page
        int i = html.indexOf("<title>") + 7;  //title of movie is title of webpage
        int j = html.indexOf("</title>");
        title = html.substring(i, j - 7);  //get title
        year = html.substring(j - 5, j - 1);  //year is always in parenthesis after the title
        //System.out.println(movieID);
        i = html.indexOf("<a href=\"/Sections/Languages"); //get the first language listed
        if (i != -1) {
            j = html.indexOf("/a>", i) + 3;
            language = html.substring(i, j);
            language = language.replaceAll("<.*?>", "").trim();
        } else {
            language = "none";
        }
        //Add line to add movie to database
        printToFile("media(" + movieID + "," + title + "," + year + "," + language + "," + type + ")\n",true);
        getProduction(html, movieID, "director");  //get the Producer(s).  Adds produced, person relationship
        getProduction(html, movieID, "writer");  //get the Writer(s).  Adds wrote, person relationship
        getCastInfo(html, movieID);  //Get the actors.  Adds actedIn, person relationship
    }


    /**
     * Get the Production people of a movie.
     * @param html the IMDB html source code for the media
     * @param movieID the movieID, e.g. tt0844286
     * @param type the type of person to add.  Either "director" or "writer"
     */
    public void getProduction(String html, String movieID, String type) {
        String heading, relation;  //heading is what to search for in the html, relation is the relation to add to the database
        if (type.toLowerCase().equals("writer")) { //We're looking for a writer
            heading = "<h5>Writer";
            relation = "wrote";
        } else if(type.toLowerCase().equals("producer")){ //We're looking for a director
            heading = "<h5>Director";
            relation = "directed";
        } else{ //Bad input
            System.out.println("Error! Unknown Production type: "+type);
            return;
        }
        int h = html.indexOf(heading);  //Find the name(s)
        h = html.indexOf("<a href=\"/name", h);
        int k = html.indexOf("</div>", h);
        while (h != -1 && h < k) { //Loop through names
            String sarr[] = getPerson(html, h, k);
            if (sarr != null) {
                printToFile(relation + "(" + sarr[0] + "," + movieID + ")\n",true);
            }
            h = html.indexOf("<a href=\"/name", h + 1); //Find index of next name
        }
    }


    /**
     * Get the information about cast members.
     * @param html  The IMDB source code for a movie
     * @param movieID  The IMDB movie id, e.g. tt0844286
     */
    public void getCastInfo(String html, String movieID) {
        int start = html.indexOf("<table class=\"cast\"");  //Cast members start here
        int end = html.indexOf("/table", start);  //Cast members end here
        int h = 0;  //The start index of a particular person
        String nameID = "", name = "";
        while (h != -1 && h < end) {  //Make sure we're only looking at cast members
            h = html.indexOf("<td class=\"nm\"", h + 1);  //Get the start index of the next person
            int k = html.indexOf("/tr>", h);  //No information about this person goes beyond here
            String[] sarr = getPerson(html, h, k);  //Get the person's name and id
            if (sarr == null) {  //An error occurred, don't add this actedIn relationship to the database
                continue;  //move on
            }
            nameID = sarr[0];  //get the id
            name = sarr[1]; //get the name
            h = html.indexOf("<td class=\"char\">", h); //character information starts here
            int i = html.indexOf(">", h) + 1;  //Character name should be around here
            int j = html.indexOf("</td>", i);  //End of character name should be around here
            String s = html.substring(i, j);
            s = s.replaceAll("<.*?>", "");  //Remove any html tags
            s = s.replaceAll("\\(.*?\\)", "");  //Remove anything in parenthesis
            sarr = s.split(" / ");  //When multiple roles are played, they're separated this way
            for (int a = 0; a < sarr.length; a++) {  //Loop through all character names
                printToFile("playedCharacter(" + nameID + "," + movieID + "," + sarr[a] + ")\n",true);
            }
        }
    }

    /**
     * Get the name and id of a person.
     * @param html the IMDB html source code for a person
     * @param h the index to start looking for this person at
     * @param k the index to stop looking for information about this person
     * @return 2 item array.  Item 0 is the id, item 1 is the name.  Null if error occurred
     */
    public String[] getPerson(String html, int h, int k) {
        String nameID, name;
        int i = html.indexOf("/name/", h);  //id should be right after here
        int j;
        if (i < k && i != -1) {  //make sure i was found and in the right place
            i += 6;  //get person's id
            j = html.indexOf("/", i);
            nameID = html.substring(i, j);
            if (!names.contains(nameID) && !researchedNames.contains(nameID)) {  //add them to research list if not already there
                if (addPerson(nameID)) {  //try to add this person to database list
                    names.add(nameID);  //If successful, also add their name to the names list
                } else {
                    return null;  //Could not add the person, return null
                }
            }
        } else {
            return null;  //id not found, error
        }

        i = html.indexOf(">", i) + 1;  //find the person's name
        j = html.indexOf("</a>", i);
        if (i == -1 || j == -1) {
            return null;  //could not find their name, error
        }
        name = html.substring(i, j);
        String[] sarr = {nameID, name};
        return sarr;
    }

    /**
     * Add more movies to the movies list.  Take a name off the names list,
     * search their IMDB page, and get movie titles (ignore non-movies).
     */
    public void findMovies() {
        String s = names.remove(0);  //Get a name.  Could be randomized
        researchedNames.add(s);  //Make sure we don't look twice
        String html = getHTML(s);  //Get the html for this person
        if (html == null) {  //Error occurred
            return;
        }
        int i = html.indexOf("/title/");  //A movie ID should be around here
        while (i != -1) {  //While there are still titles to be examined
            i += 7;
            //Look in the movie list (titles will be in list tags)
            int h = html.lastIndexOf("<li", i);
            int k = html.lastIndexOf("</li>", i);
            if (k > h) { //make sure we're looking at a movie inside a list tage
                i = html.indexOf("/title/", i + 1);  //Move on if not
                continue;
            }
            int j = html.indexOf("/", i); //get movieID
            String movieID = html.substring(i, j);
            i = html.indexOf("</a>", i) + 4;
            j = html.indexOf("</li>", i);
            String info = html.substring(i, j); //Get information that comes after the movie to make sure we're actualy getting a movie (as opposed to a video game or tv show or something)
            if (!(info.contains("(V)") || info.contains("(TV)") ||info.contains("TV episode") || info.contains("details only on") || info.contains("in production")) && info.matches(".*\\(\\d\\d\\d\\d\\).*")) {
                j = html.indexOf(")", i);
                if (!(researchedTitles.contains(movieID) || titles.contains(movieID))) { //add movie to movie list if not there already
                    titles.add(movieID);
                }
            }
            i = html.indexOf("/title/", i + 1); //move on to next movie title
        }
    }

    /**
     * Print a string to a file.  Intended to be an SQL command
     * @param s string to print.  Should be an SQL command.  Command should keep
     * database internally consistent.
     * @param append True to append to the output file, false to overwrite
     */
    public void printToFile(String s, boolean append) {
        try {
            FileWriter fstream = new FileWriter("out.txt", append);
            BufferedWriter out = new BufferedWriter(fstream);
            out.write(s);
            out.close();
        } catch (Exception e) {//Catch exception if any
            System.err.println("Error: " + e.getMessage());
        }
    }
}
